package edu.hawaii.webspider;

import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;
import com.meterware.httpunit.WebLink;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashSet;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;

import java.util.logging.Logger;
import org.xml.sax.SAXException;
import com.meterware.httpunit.HttpNotFoundException;
import static java.lang.System.out;


/**
 * A class to count the total links on a page.
 * @author Kevin English (with support from Phillip Johnson)
 */
public abstract class WebAnalysisTool  {

  protected static final Logger logger = Logger.getLogger("edu.hawaii.webspider");
  /**
   * Contains the starting url for the crawl. 
   */
  protected String startUrl = "";
  /**
   * Contains a map of web responsew with the url as their key. 
   */
  protected Map<String,WebResponse> webResponses = new HashMap<String,WebResponse>();
  /**
   * Contains a web conversation object. 
   */  
  protected WebConversation webConversation;
  /**
   * A constructor that takes no arguments.
   * 
   */
  public WebAnalysisTool() {
    logger.info(" WebAnalysisTool constructor called ");
    turnJavaScriptOff();
    this.webConversation = new WebConversation();    
  }
  
  /**
   * A constructor that takes an argument for the site that will be crawled.
   * 
   * @param startUrl The home url for the crawl.
   */
  
  public WebAnalysisTool(String startUrl) {
    this.setStartUrl(startUrl);
    turnJavaScriptOff();
    this.webConversation = new WebConversation();    
  }
  
  
  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   */
  protected abstract boolean isStopCondition();   
  
  /**
   * Getter method for starturl.
   * @return start url
   */  
  public final String getStartUrl() {
    return this.startUrl;
  }
  
  /**
   * Setter method for StartUrl variable. 
   * This variable is where you will start you web crawling from.
   * 
   * @param startUrl The home url for the crawl.
   */
  public final void setStartUrl(String startUrl) {
    this.startUrl = startUrl;
  }

  /**
   * Add links to the webResponses hash from a response object. Call addLink.
   * @param response A response object from which the links will be added.
   */
  protected void processResponseLinks(WebResponse response)   {
    try {
      for (WebLink link : response.getLinks()) {
        processLink(link);
        if (this.isStopCondition()) {
          break;
        }
      }
    }
    catch (SAXException e) {
      e.printStackTrace();
    } 
    catch (Exception e) {        
      e.printStackTrace();      
    }
  }
  
  /**
   * Add a single link to the webResponses hash.
   *
   * @param link A Valid WebLink Object
   * @return webresponse A WebResponse object or null if the link can not be reached.
   * 
   */
    
   
  protected WebResponse processLink(WebLink link)  {
    String urlString   = getUrlString(link);
    if (urlString == null) {
      return null;
    }
    try {
      // although the hash ensures that 2 object will not be added twice
      // we make sure it doesn't already exist to save the time it takes 
      // to load the page when we call link.click().
      if (this.webResponses.get(urlString) == null ) {
        WebResponse response = link.click();
        if (response.getContentType().equals("text/html")) {
          this.webResponses.put(urlString, response);
          out.print(".");  
          return response;
        }
      } 
    } 
    catch (HttpNotFoundException e) {
      return null;
    } 
    catch (java.net.UnknownServiceException e ) {
      return null;
    } 
    catch (java.lang.RuntimeException e) {
      return null;
    }
    catch (Exception e) {        
      return null;      
    }
    return null;
      
  }  
  
  /**
   * Returns the full url string for a weblink object.
   * @param link A WebLink object.
   * @return A string with the url
   */
  protected String getUrlString(WebLink link) {
    try {
      return link.getRequest().getURL().toString();
    }
    catch (MalformedURLException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
      return null;
    }
  }
  /**
   * Turns off the Javascript exceptions.
   * http://osdir.com/ml/web.httpunit.devel/2006-07/msg00000.html
   *
   */
  public static void turnJavaScriptOff() {
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);    
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);
  }

  /**
   * Validates a url.
   * @param url eg http://www.google.com
   * @return true if url valid
   */
  
  public static boolean isValidHostUrl(String url) {
    turnJavaScriptOff();
    try {
      WebConversation wc = new WebConversation();
      WebRequest request = new GetMethodWebRequest(url);
      wc.getResponse(request);
    } 
    catch (java.net.UnknownHostException e) {
      //e.printStackTrace();
      return false;      
    }
    catch (MalformedURLException e) {
      // TODO Auto-generated catch block
      //e.printStackTrace();
      return false;      
    }
    catch (IOException e) {
      // TODO Auto-generated catch block
            
      //e.printStackTrace();
      return false;      
    }
    catch (SAXException e) {
      // TODO Auto-generated catch block
//      e.printStackTrace();
      return false;      
    }
    catch (Exception e) {
      return false ;
    }
    return true;
  }
  /**
   * Populate the web response hash set.
   * @param response A valid WebResponse object.
   */
  protected void populateWebResponses(WebResponse response)  {
    WebResponse webResponse = response;
    Set<String> exhaustedKeys = new HashSet<String>();
    webResponses.put(this.startUrl, webResponse);    
    while (!this.isStopCondition()) {
      processResponseLinks(webResponse);
      Set<String> responseKeys = webResponses.keySet();
      if (exhaustedKeys.containsAll(responseKeys)) {
        break;
      }
      Set<String> unexhaustedKeys = new HashSet<String>(responseKeys);
      unexhaustedKeys.removeAll(exhaustedKeys);
      String linkKey =   unexhaustedKeys.iterator().next();
      webResponse = webResponses.get(linkKey);
      logger.info("Crawling to " + linkKey);
      exhaustedKeys.add(linkKey);
    }
    out.println(".");
  }
  
  
}
